"""

.. module:: http_service
.. moduleauthor:: @author: 

This module contains the HTTPService class.

It returns some data and messages as text/plain. These are always prefixed with 
an identifier:

    [restart] Restart requested.
    [resource-added] Resource (%s) added to spaces.
    [resource-deleted] Resource (%s) deleted.
    [error-delete-resource-not-found] Could not delete resource (%s). Not found.
    [mapping-not-found] Mapping not found (%s)
    [mapping-not-permitted] HTTP Method (%s) is not permitted for this URL.
    [diag-template-not-found] Template requested but not available.
    [diag-unknown-parameter] unknown diag parameter (%s).
    [data] %s
    [error-file-not-found] File not found.

*Constants*:

.. seealso:: class :class:`runtime.service.Service`
"""

# Imports
#
from SocketServer import ThreadingMixIn
from datastore.resource import *
from datastore.uri import *
from runtime.node import LoggingLevels
from runtime.service import *
from services.htmltemplate import *
from services.makohttptemplate import MakoHTTPTemplate
from urlparse import urlparse
import BaseHTTPServer
import cgi
import httplib
import logging
import socket
import threading
import time
import types
import re



# Constants
#

# Classes
#
class HTTPService(Service):
    """
    **Constants**:
    """
    
    def cb_start(self, uri, action, res):
        """
        :param uri: Uri
        :param action:
        :param res: Resource
        """
        # fetch and validate the node definition resource
        node_def_uri = self.service_def.options.node_configuration
        self.node_def = self.spaces.get(node_def_uri)
        if not self.node_def.validate(self.spaces):
            self.logger.log(logging.ERROR,
                "Unable to start HTTPService. Invalid node definition.")
            return
        # ignore update in watch because we use it in this service.
        self.spaces.set_watch(Uri('internal:base/http/site/'), 
                        self.cb_add_site, filter=['put'])
        self.spaces.set_watch(Uri('internal:base/http/site/'), 
                        self.cb_delete_site, filter=['delete', 'remove'])

    def cb_stop(self, uri, action, res):
        """
        :param uri: Uri
        :param action:
        :param res: Resource
        """
        self.logger.log(logging.DEBUG, "Stopping the HTTP Service.")
        for site_uri in self.spaces.find(Uri('internal:base/http/site/')):
            self.spaces.delete(site_uri)

    def cb_add_site(self, uri, action, res):
        """
        This function adds a new HTTP server site (based on the site resource).     
        If the state variable is set, then this is an update, not a new site.
        
        :param uri: Uri
        :param action:
        :param res: Resource
        """
        # TODO: handle sites that are a subset of another site .e.g. "/admin"
        
        if res.validate(self.spaces):
            if not res.has_var('state'):
                self.logger.log(logging.DEBUG, "uri (%s)", uri)
                self.async(self.cb_run_server, uri, action, res, 
                           name="HTTP %s" % res.dc_title())
        else:
            self.logger.log(logging.ERROR,
                "Unable to start site (%s). Invalid definition.",res.dc_title())
            res.state = 'error'
            self.spaces.put(uri, res)
            return
    
    def cb_run_server(self, uri, action, res):
        """
        Runs asynchronously, one thread per site.
        
        :param uri: Uri
        :param action:
        :param res: Resource
        """
        # get the ip address or domain name and port to listen on
        #self.spaces.set_trace(uri)
        port = self.node_def.nodeportrangelow + res.port_offset
        if port > self.node_def.nodeportrangehigh:
            self.logger.log(logging.ERROR,
                "Unable to start site (%s). Port number too high (%s > %s)", 
                res.dc_title(), port, self.node_def.nodeportrangehigh)
            res.state = 'error'
            self.spaces.put(uri, res)
            return
        res.port = port
        ip = self.node_def.nodeipaddress
        self.logger.log(logging.INFO, "Starting site (%s) on port (%s)", 
                        res.dc_title(), port)
        try:
            res.state = 'starting'
            self.spaces.put(uri, res)
            httpd = Server((ip, port), Handler,
                        self.spaces, uri, self.node_def, self.service_def)
        except socket.error, (value, message): 
            if value == 48:
                self.logger.log(logging.ERROR, "Unable to start site " + \
                    "(%s) at address %s:%s. (%s)", 
                    res.dc_title(), ip, port, message)
            res.state = 'error'
            self.spaces.put(uri, res)
            return
        except:
            self.logger.exception("error (%s)", sys.exc_info()[1])
            res.state = 'error'
            self.spaces.put(uri, res)
            return

        res.state = 'running'
        res.server = httpd
        # server is not a standard Resource object type so use copy=False
        self.spaces.put(uri, res, copy=False)
        httpd.serve_forever()
             
    def cb_delete_site(self, uri, action, res):
        """
        This function deletes a site currently being listened on.
        
        :param uri: Uri
        :param action:
        :param res: Resource
        """
        self.logger.log(logging.DEBUG, "Shutdown HTTP server (%s)", uri)
        # we want the non-copied site resource (with the server reference)
        res = self.spaces.get(uri, copy=False)
        if res.state == 'running':
            res.state = 'stopping'
            res.server.shutdown()


class Server(ThreadingMixIn, BaseHTTPServer.HTTPServer):
    """
    **Constants**:
    
    :param server_address:
    :param RequestHandler:
    :param spaces:
    :param site_uri:
    :param node_definition_res: The global node definition resource    
    :param service_def: The http_service service definition resource
    """
    # http://docs.python.org/library/socketserver.html#module-SocketServer
    allow_reuse_address = True
    #request_queue_size = 5
    
    def __init__(self, server_address, RequestHandler,
                 spaces, site_uri, node_definition_res, service_def):  
        self.logger = logging.getLogger(
            'yarra.services.httpserver')
        self.spaces = spaces
        self.site_uri = site_uri
        self.site_res = spaces.get(site_uri)
        self.node_definition_res = node_definition_res
        self.service_def = service_def
        BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandler)

class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
    """
    Ref: http://wiki.python.org/moin/BaseHttpServer
    
    **Constants**:
    
    :param server_address:
    :param RequestHandler:
    :param spaces:
    :param site_uri:
    :param node_definition_res:         
    """
    
    def do_HEAD(self):
        """
        For now, same as GET. Performance Improvement possibilities here,
        especially if a cache is included for GET but more important tasks 
        to complete for the moment.
        """
        self.do_GET()
        return
        
    def do_GET(self):
        """
        Handle HTTP GET. Resources are fetched from spaces.
        
        A special URL is processed immediately to hand node restart, which means
        it will probably function, even where there are problems in other 
        area's.
        """
        if not self.check_path_valid_for_site(self.path, self.server.site_res):
            self.send_http_response(httplib.NOT_FOUND)
            return
        
        # FIXME: Add some basic security here.
        
        # Handle the special case first.
        if self.path == "/admin/restart":
            # and restart the node
            control_res = Resource(type='term:base/runtime/node_control')
            control_res.action = 'restart'
            self.server.spaces.put(Uri('internal:base/node_control'),
                                   control_res)
            self.send_response(200)
            self.send_header("Content-type", "text/plain")
            self.end_headers()
            self.wfile.write("[restart] Restart requested.")
            return
        
        self.url = urlparse(self.path)
        req_uri = self.server.site_res.uri_prefix + self.url.path[1:]
        self.mapping = self.find_mapping(self.server.spaces, self.server.service_def, req_uri, "GET")
        if self.mapping is not None:
            req_uri = self.mapping['uri']
        self.save_query()   # always sets self.query
        self.form_fields = None
        self.form_data =  None
        self.form_content_type = None
        if len(self.query) != 0:
            # The req_uri is for the response to the put.
            req_uri = self.process_post_and_get_query(req_uri)      
        if req_uri is not None:
            self.process_get(req_uri)
        return
    
    def do_POST(self):
        """
        """
        if not self.check_path_valid_for_site(self.path, self.server.site_res):
            self.send_http_response(httplib.NOT_FOUND)
            return        
        self.url = urlparse(self.path)
        self.save_query()   # always sets self.query
        # always sets self.form_fields, self.form_data, self.form_content_type
        self.save_data()
        req_uri = self.server.site_res.uri_prefix + self.url.path[1:]
        self.mapping = self.find_mapping(self.server.spaces, self.server.service_def, req_uri, "POST")
        if self.mapping is not None:
            req_uri = self.mapping['uri']
        req_uri = self.process_post_and_get_query(req_uri)
        # returns None if there was an error and a response has been sent.
        if req_uri is not None:
            self.process_get(req_uri)
        return
    
    def do_PUT(self):
        """
        PUT a new resource into spaces. For example::
        
            PUT /admin/spaces/external:hello HTTP/1.0
            Content-Length: 40
            <resource><content></content></resource>
        """
        if not self.check_path_valid_for_site(self.path, self.server.site_res):
            self.send_http_response(httplib.NOT_FOUND)
            return        
        self.url = urlparse(self.path)
        req_uri = self.server.site_res.uri_prefix + self.url.path[1:]
        self.mapping = self.find_mapping(self.server.spaces, self.server.service_def, req_uri, "PUT")
        if self.mapping is not None:
            req_uri = self.mapping['uri']
        # fetch the body
        input_stream = self.rfile
        content_length = int(self.headers['content-length'])
        body = input_stream.read(content_length)
        res = Resource(body)
        self.server.spaces.put(req_uri, res)
        m = "[resource-added] Resource (%s) added to spaces." % req_uri
        self.server.logger.log(logging.INFO, m)
        self.send_http_response(200, req_uri,
                    'text/plain', len(m), m)
        return
    
    def do_DELETE(self):
        """
        DELETE will delete a resource from spaces. For example::
        
            DELETE /admin/spaces/dc:title HTTP/1.0
        """
        if not self.check_path_valid_for_site(self.path, self.server.site_res):
            self.send_http_response(httplib.NOT_FOUND)
            return        
        self.url = urlparse(self.path)
        req_uri = self.server.site_res.uri_prefix + self.url.path[1:]
        self.mapping = self.find_mapping(self.server.spaces, self.server.service_def, req_uri, "DELETE")
        if self.mapping is not None:
            req_uri = self.mapping['uri']
        if self.server.spaces.exists(req_uri):
            self.server.spaces.delete(req_uri)
            m = "[resource-deleted] Resource (%s) deleted." % req_uri
            self.server.logger.log(logging.INFO, m)
            self.send_http_response(200, req_uri,
                        'text/plain', len(m), m)
        else:
            m = "[error-delete-resource-not-found] " + \
                "Could not delete resource (%s). Not found." % req_uri
            self.server.logger.log(logging.INFO, m)
            self.send_http_response(404, req_uri,
                            'text/plain', len(m), m)
        return
    
    def save_query(self):
        self.query = {}
        for part in self.url.query.split('&'):
            if len(part) > 0:
                key, value = part.split('=')
                self.query[key] = value
                
        # Save and then remove the 'diag' query string, if present
        if 'diag' in self.query.keys():
            self.diag = self.query['diag']
            del self.query['diag']
        else:
            self.diag = None

    def save_data(self):
        """
        Parse the form data posted.
        ref: http://docs.python.org/library/cgi.html
        
        This code doesn't handle the general case of multiple attachments, such
        as 'multipart/*'.
        
        FIXME: This code needs some tidying-up
        """
        self.form_fields = {}
        self.form_data = None
        self.form_content_type = self.headers['Content-Type']
        self.server.logger.log(logging.INFO,"Form content type (%s)",
                               self.form_content_type)
        if ('multipart/form-data' in self.form_content_type.lower()) or \
           ('application/x-www-form-urlencoded' in self.form_content_type.lower()):
            form = cgi.FieldStorage(
                fp=self.rfile,
                headers=self.headers,
                environ={'REQUEST_METHOD':'POST',
                         'CONTENT_TYPE': self.form_content_type})
            for key in form.keys():
                if type(form[key]) == types.ListType:
                    element_list = []
                    for element in form[key]:
                        element_list.append(element.value)
                    self.form_fields[key] = element_list
                else:
                    self.form_fields[key] = form[key].value
        elif '/xml' in self.form_content_type.lower():
            # this covers text/xml and application/xml
            self.server.logger.log(logging.INFO,"Form content length (%s)",
                               self.headers['Content-Length'])    
            input_stream = self.rfile
            content_length = int(self.headers['content-length'])
            self.form_data = input_stream.read(content_length)

    def find_mapping(self, spaces, service_def, req_uri, http_method):
        """
        The uri 'internal:base/http/url_mapping/' is a prefix for mapping
        resources.
        The first matching URL is used.
        """
        prefix = service_def.options.mapping_prefix
        for res_uri in spaces.find(prefix):
            res = spaces.get(res_uri)
            for map_uri in res.map.keys():
                if Uri(map_uri) == req_uri and http_method in res.map[map_uri].http_methods:
                    return res.map[map_uri]
        return None

    def process_post_and_get_query(self, req_uri):
        """
        Processing for post:
        
            #.  Check if a 'term:base/services/http_service/post_def' resource
                exists at the specified (non-collection) uri.
            #.  Perform a Spaces request() to the post collection. A callback
                will be specified to remove the response, which is passed to 
                process_response() to handle the response page - the same as a
                GET response would be handled.
            #.  Clean-up the request Resource.
        
        A POST results in a 'term:base/services/http_service/post' resource
        being put to the request queue.
        """
        post_res = Resource(type='term:base/services/http/post')
        post_res.query_data = self.query
        post_res.form_data = Data(self.form_data)
        post_res.form_fields = self.form_fields
        # FIXME: we also have self.form_content_type but its not cleaned yet
        if self.form_data is not None:
            self.server.logger.log(logging.INFO,"XML (len %i)\n%s", 
                               len(self.form_data), self.form_data)
        
        # Check that we have a valid mapping.
        if self.mapping is None:
            text = "[mapping-not-found] Mapping not found (%s)" % req_uri
            self.server.logger.log(logging.DEBUG, text)
            self.send_http_response(404, req_uri,
                "text/plain", len(text), text)
            return None
        elif self.command not in self.mapping.http_methods:
                text = "[mapping-not-permitted] HTTP Method " + \
                    "(%s) is not permitted for this URL." % self.command
                self.server.logger.log(logging.DEBUG, text)
                self.send_http_response(404, req_uri,
                    "text/plain", len(text), text)
                return None

        # Make the request (and wait for a response, well, just wait!)
        self.semaphore = None
        self.response_posted = False
        self.server.logger.log(logging.DEBUG, "Request (%s) %s",
                               req_uri, self.command)
        new_uri = self.server.spaces.request(req_uri, post_res, 
                                   self.cb_post_response)
        # Create our semaphore and wait until our callback fires (unless it
        # already has).
        if not self.response_posted:
            self.server.logger.log(logging.DEBUG,
                                "[semaphore] Wait for callback")
            # Semaphore doesn't block the first time acquire is called, you
            # need to acquire it a second time to block execution. A single
            # release will continue execution.
            self.semaphore = threading.Semaphore()
            self.semaphore.acquire(True)
            self.semaphore.acquire(True)
        return self.response_callback_uri
        
    def cb_post_response(self, uri, action, res):
        self.server.logger.log(logging.DEBUG, "[semaphore] callback (%s %s)",
                               str(uri), action)
        # If the response is also saved to spaces, then this callback will
        # fire twice. Just ignore the second time.
        if not self.response_posted:
            self.response_posted = True
            
            # save our data
            self.response_callback_uri = uri
            self.response_callback_action = action
            self.response_callback_res = res
            
            # continue processing in post_post (unless we beat the wait)
            if self.semaphore is not None:
                self.semaphore.release()
        
    def process_get(self, req_uri):
        """
        This method is used for any HTTP method that returns a response.
        
        If the content has already been generated by a service, then the 
        resource will be of type 'term:base/services/http/response'. In 
        this case, the resource content and attributes are used as specified.
        No further template is applied. this will generally be the case for
        images and other non-html output.
        
        If a resource is returned, and its definition resource contains a
        variable 'template', then this is assumed to be a template that should
        be applied along with the site template.
        
        In all other cases, plain text is specified if a fragment is specified 
        (as a parameter), or the raw resource XML is returned.
        
        Query parameters are processed as follows:
        
            - diag=raw - force the return of the resource as XML.
            - diag=template - force the return of the template as XML.
            
        If there are other query parameters present then the request is 
        processed like a POST so, that the query data is available to the
        service that ahdnles this Request.
        """
        
        self.server.logger.log(logging.INFO, "Spaces get (%s) %s", 
                               req_uri, self.url.params)
        if self.server.spaces.exists(req_uri):
            try:
                # Copy=False should prevent a deserialisation exception
                res = self.server.spaces.get(req_uri, copy=False)
            except:
                m = "[unknown-exception] Resource could not be displayed."
                self.server.logger.log(logging.INFO, m)
                self.send_http_response(200, req_uri,
                                        'text/plain', len(m), m)
                return
                
            self.server.logger.log(logging.DEBUG, "resource type (%s)",
                               str(self.query))
            # get the definition resource if it exists
            def_res = None
            template_res = None
            if res.get_type() is not None:
                if self.server.spaces.exists(res.get_type()):
                    def_res = self.server.spaces.get(res.get_type())
                    if def_res.get_var('template', '') != '':
                        template_res = self.server.spaces.get(def_res.template)
            # support url?diag=raw and url?diag=template
            if self.diag is not None:
                if self.diag == 'raw':
                    self.send_http_response(200, req_uri,
                        "application/xml", len(str(res)), str(res))
                    return
                elif self.diag == 'template':
                    if template_res is not None:
                        self.send_http_response(200, req_uri,
                            "application/xml", len(str(template_res)), 
                            str(template_res))
                        return
                    else:
                        m = "[diag-template-not-found] " + \
                            "Template requested but not available."
                        self.server.logger.log(logging.INFO, m)
                        self.send_http_response(200, req_uri,
                                            'text/plain', len(m), m)
                        return
                else:
                    m = "[diag-unknown-parameter] " + \
                        "unknown diag parameter (%s)." % self.diag
                    self.server.logger.log(logging.INFO, m)
                    self.send_http_response(200, req_uri,
                                            'text/plain', len(m), m)
                    return

            # Check if we have been passed HTML ready for the application of the
            # site template.
            if res.get_type() == Uri('term:base/services/http/response'):
                self.server.logger.log(logging.DEBUG, "response (%s)", 
                               str(res.dc_title(default='')))
                if res.get_var('status', '') == '':
                    status = 200
                else:
                    status = int(res.status)
                self.send_http_response(status, req_uri,
                    res.content_type, res.content_length, res.content, res.get_var('headers', {}))
                return
            # Use the template if one is specified in the definition resource.
            if def_res is not None:
                # If a template is specified, then apply it.
                if template_res is not None:
                    template_type = template_res.get_metadata(Uri("term:base/service/http/template_type"), "Default")
                    # The template engine we use is based on the type of the template
                    if template_type == "Mako":
                        site_template_res = self.server.spaces.get(
                                        self.server.site_res.base_mako_template_uri)
                        x = MakoHTTPTemplate(self.server.spaces, site_template_res, template_res, res)
                        html = x.render().strip()
                    else:
                        site_template_res = self.server.spaces.get(
                                        self.server.site_res.base_default_template_uri)                        
                        x = HTMLTemplate(self.server.spaces, site_template_res,
                                        template_res, res, verbose=True)
                        html = x.generate().strip()
                    self.send_http_response(200, req_uri,
                        'text/html', len(html), html)
                    return
            # If there was no parameter, return the raw XML, otherwise
            # return the requested fragment from the resource as plain text.
            #
            # Note: we are using a parameter (;), not a fragment (#), 
            # because the fragment is not passed by the browser.
            # e.g. http://a.com/fred;param vs. http://a.com/fred#param
            if self.url.params == '':
                self.send_http_response(200, req_uri,
                    "application/xml", len(str(res)), str(res))
                return
            else:
                part = "[data] %s" % res.get_var_part(self.url.params)
                self.send_http_response(200, req_uri,
                    "text/plain", len(str(part)), str(part))
                return      
        else:
            text = "[error-file-not-found] File not found."
            self.server.logger.log(logging.DEBUG, text)
            self.send_http_response(404, req_uri,
                "text/plain", len(text), text)
            return

    def log_message(self, format, *args):
        """ 
        Override default method. 
        
        :param format:
        :param args:        
        """
        self.server.logger.log(logging.INFO, args[0])

    def send_http_response(self, http_status, request_uri=None, 
                           content_type=None, 
                           content_length=0, data=None, headers=None):
        """
        This function creates the HTTP message and sends it to the client.
        
        :param http_status: int - HTTP status code
        :param request_uri:
        :param content_type: String - HTTP "Content-Type:".
        :param content_length: int - HTTP "Content-Length:".
        :param data: String - output.
        :param headers:
        """
        try:
            self.send_response(http_status)
            self.send_header('Cache-Control', 'no-cache')
            if (content_type is not None):
                self.send_header('Content-Type', content_type)
            self.send_header('Content-Length', content_length)
            if headers is not None:
                for header_key, header_value in headers.items():
                    self.send_header(header_key, header_value)
            self.end_headers()
            if (data is not None):
                if type(data) == type(Data()):
                    self.server.logger.log(logging.INFO, "Data().")
                    self.wfile.write(data.decode())
                else:
                    self.wfile.write(data)
        except socket.error, (value, message): 
            #if value == 48:
            self.server.logger.log(logging.ERROR, "Unable to send response " + \
                " to (%s): %s %s", request_uri, value, message)
        finally:
            if self.wfile is not None:
                self.wfile.close()
        self.server.logger.log(logging.INFO, "%s %s %s", 
            self.command, str(http_status), request_uri)
        self.server.logger.log(LoggingLevels.TRACE, "content (%i):\n%s", 
                               content_length, data)
        # Make sure the text is sent before we drop the connection.
        if http_status != 200:
            time.sleep(0.1)

    def check_path_valid_for_site(self, path, site_res):
        """
        Checks that the path is valid for this site, by checking the path against the valid list of paths in the 
        site resource.
        
        :param path: Path of current request.
        :param site_res: Site resource.
        :rtype: True or False
        """           
        for valid_paths_pattern in site_res.valid_paths_patterns:
            matches = re.match(valid_paths_pattern, path)
            if matches:
                return True
        return False
        
    def authentication(self, url):
        """
        Returns a resource that includes the account information and a boolean
        to indicate if access should be allowed.
        
        Need to pass credentials to be validated.
        
        Supports basic authentication and form based authentication using 
        session information.
        
        :param url:
        """
        return True

    def session(self, url):
        """
        Extracts cookie or URL based token and returns a session resource for
        a new or existing session.
        
        :param url:
        """
        return
    
    def clean(self, str, extra='', fail='exception'):
        """
        Raise an exception or return None if an invalid character is in the
        string.
        
        This is used to filter data input from URL's or post data that may
        be appended to a Uri or used in some other dangerous way. User names
        are an example of data that should be cleaned.
        
        Permitted characters are::
        
            a-zA-z0-9.-_
        
        These characters are dangerous in a Uri::
        
            #/:;
        
        Note: This method is NOT SUITABLE for cleaning file names.
        
        :param str:
        :param extra:
        :param fail:
        """
        return str # Not very clean so far!!!!!!!
